{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00,  1.13it/s]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import tqdm\n",
    "import random\n",
    "import numpy as np\n",
    "import scipy.stats\n",
    "from statsmodels.distributions.empirical_distribution import ECDF\n",
    "\n",
    "TEST_FRACTION = 0.2\n",
    "\n",
    "def js_distance(P, Q):\n",
    "    overall_min = min(P.min(), Q.min())\n",
    "    overall_max = max(P.max(), Q.max())\n",
    "    kP = scipy.stats.gaussian_kde(P)\n",
    "    kQ = scipy.stats.gaussian_kde(Q)\n",
    "    xs = np.linspace(overall_min, overall_max, 100)\n",
    "    Psmooth = kP(xs)\n",
    "    Qsmooth = kQ(xs)\n",
    "    Msmooth = (Psmooth + Qsmooth) / 2\n",
    "    KL_PM = scipy.stats.entropy(Psmooth, Msmooth)\n",
    "    KL_QM = scipy.stats.entropy(Qsmooth, Msmooth)\n",
    "    return 0.5 * (KL_PM + KL_QM)\n",
    "\n",
    "data_dir = os.path.normpath(os.path.join(sys.path[0], '../data/'))\n",
    "basenames = os.listdir(os.path.join(data_dir, 'interim', 'X'))\n",
    "num_test = int(len(basenames) * TEST_FRACTION)\n",
    "num_train = len(basenames) - num_test\n",
    "labels = ['test'] * num_test + ['train'] * num_train\n",
    "\n",
    "# Pick a train-test split such that the training and testing distributions look\n",
    "# as similar as possible. With a much larger amount of data, this would be\n",
    "# unnecessary; however, there are only a fairly small number of seizures, and so\n",
    "# we can't necessarily rely on random sampling alone.\n",
    "\n",
    "distances = []\n",
    "for i in tqdm.trange(1):\n",
    "    random.shuffle(basenames)\n",
    "    Ys_clustered = {'train': [], 'test': []}\n",
    "    for (kind, file) in zip(labels, basenames):\n",
    "        Ys_original = np.load(os.path.join(data_dir, 'interim', 'Y', file))\n",
    "        Ys_clustered[kind] += list(Ys_original)\n",
    "    train_dist = np.array(Ys_clustered['train'])\n",
    "    test_dist = np.array(Ys_clustered['test'])\n",
    "    distances.append((js_distance(train_dist, test_dist), basenames))\n",
    "\n",
    "basenames = min(distances)[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.debugger import set_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_features = np.concatenate([\n",
    "    np.load(os.path.join(data_dir, 'interim', 'X', basename))\n",
    "    for basename in basenames\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "cdfs = [ECDF(values) for values in all_features.T]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize_feature_vectors(Xs_original):\n",
    "    Xs_transformed = np.full_like(Xs_original, np.nan)\n",
    "    for (i, (x, ecdf)) in enumerate(zip(Xs_original.T, cdfs)):\n",
    "        # The nudging is to prevent the highest/lowest value from getting a\n",
    "        # quantile of exactly +/- 1, which would have a z-score of +/- infinity\n",
    "        x_nudged = x + 0.000000001 * np.ptp(x) * np.sign(np.median(x) - x)\n",
    "        x_transformed = scipy.stats.norm.ppf(ecdf(x_nudged))\n",
    "        if not np.isfinite(x_transformed).all():\n",
    "            set_trace()\n",
    "        Xs_transformed[:, i] = x_transformed\n",
    "    return Xs_transformed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "chb12_42_0.npy\n",
      "chb18_31_0.npy\n",
      "chb14_18_0.npy\n",
      "chb13_19_0.npy\n",
      "chb18_30_0.npy\n",
      "chb23_08_1.npy\n",
      "chb19_28_0.npy\n",
      "chb12_42_3.npy\n",
      "chb13_40_0.npy\n",
      "chb12_42_4.npy\n",
      "chb06_18_0.npy\n",
      "chb06_04_0.npy\n",
      "chb18_32_0.npy\n",
      "chb04_28_0.npy\n",
      "chb03_01_0.npy\n",
      "chb15_40_0.npy\n",
      "chb04_08_0.npy\n",
      "chb05_22_0.npy\n",
      "chb02_19_0.npy\n",
      "chb10_89_0.npy\n",
      "chb10_12_0.npy\n",
      "chb12_38_2.npy\n",
      "chb06_01_0.npy\n",
      "chb20_13_1.npy\n",
      "chb23_09_1.npy\n",
      "chb08_13_0.npy\n",
      "chb13_40_1.npy\n",
      "chb17a_04_0.npy\n",
      "chb15_54_4.npy\n",
      "chb22_38_0.npy\n",
      "chb15_31_0.npy\n",
      "chb09_06_0.npy\n",
      "chb15_49_0.npy\n",
      "chb12_08_2.npy\n",
      "chb10_27_0.npy\n",
      "chb04_05_0.npy\n",
      "chb13_55_0.npy\n",
      "chb09_19_0.npy\n",
      "chb12_38_3.npy\n",
      "chb21_19_0.npy\n",
      "chb15_28_0.npy\n",
      "chb08_05_0.npy\n",
      "chb12_38_1.npy\n",
      "chb12_08_3.npy\n",
      "chb12_09_1.npy\n",
      "chb22_25_0.npy\n",
      "chb01_04_0.npy\n",
      "chb23_08_0.npy\n",
      "chb08_02_0.npy\n",
      "chb16_10_0.npy\n",
      "chb11_99_0.npy\n",
      "chb13_60_0.npy\n",
      "chb19_29_0.npy\n",
      "chb09_08_1.npy\n",
      "chb21_21_0.npy\n",
      "chb17a_03_0.npy\n",
      "chb13_62_0.npy\n",
      "chb20_16_0.npy\n",
      "chb13_59_0.npy\n",
      "chb02_16+_0.npy\n",
      "chb06_13_0.npy\n",
      "chb06_09_0.npy\n",
      "chb19_30_0.npy\n",
      "chb12_06_1.npy\n",
      "chb03_04_0.npy\n",
      "chb15_40_2.npy\n",
      "chb05_16_0.npy\n",
      "chb20_15_0.npy\n",
      "chb12_11_0.npy\n",
      "chb09_08_0.npy\n",
      "chb15_54_0.npy\n",
      "chb06_01_2.npy\n",
      "chb20_12_0.npy\n",
      "chb20_14_0.npy\n",
      "chb16_17_0.npy\n",
      "chb07_12_0.npy\n",
      "chb16_18_0.npy\n",
      "chb03_36_0.npy\n",
      "chb18_29_0.npy\n",
      "chb13_58_0.npy\n",
      "chb12_42_2.npy\n",
      "chb16_14_0.npy\n",
      "chb15_52_0.npy\n",
      "chb08_11_0.npy\n",
      "chb05_06_0.npy\n",
      "chb15_54_2.npy\n",
      "chb12_42_1.npy\n",
      "chb16_17_1.npy\n",
      "chb14_03_0.npy\n",
      "chb16_18_1.npy\n",
      "chb20_15_1.npy\n",
      "chb01_03_0.npy\n",
      "chb03_34_0.npy\n",
      "chb24_04_0.npy\n",
      "chb12_38_0.npy\n",
      "chb05_13_0.npy\n",
      "chb11_92_0.npy\n",
      "chb12_10_1.npy\n",
      "chb06_10_0.npy\n",
      "chb14_04_0.npy\n",
      "chb11_82_0.npy\n",
      "chb12_38_4.npy\n",
      "chb24_03_0.npy\n",
      "chb10_20_0.npy\n",
      "chb06_01_1.npy\n",
      "chb12_23_2.npy\n",
      "chb18_35_0.npy\n",
      "chb04_28_1.npy\n",
      "chb15_54_3.npy\n",
      "chb14_11_0.npy\n",
      "chb01_21_0.npy\n",
      "chb12_08_0.npy\n",
      "chb15_22_0.npy\n",
      "chb23_06_0.npy\n",
      "chb03_35_0.npy\n",
      "chb12_23_0.npy\n",
      "chb17b_63_0.npy\n",
      "chb18_36_0.npy\n",
      "chb03_03_0.npy\n",
      "chb12_06_0.npy\n",
      "chb13_21_0.npy\n",
      "chb07_13_0.npy\n",
      "chb21_22_0.npy\n",
      "chb14_17_0.npy\n",
      "chb13_55_1.npy\n",
      "chb21_20_0.npy\n",
      "chb06_24_0.npy\n",
      "chb15_06_0.npy\n",
      "chb07_19_0.npy\n",
      "chb01_15_0.npy\n",
      "chb15_20_0.npy\n",
      "chb05_17_0.npy\n",
      "chb15_40_1.npy\n",
      "chb08_21_0.npy\n",
      "chb02_16_0.npy\n",
      "chb12_09_0.npy\n",
      "chb15_10_0.npy\n",
      "chb15_62_0.npy\n",
      "chb13_62_1.npy\n",
      "chb23_09_2.npy\n",
      "chb23_09_3.npy\n",
      "chb14_04_1.npy\n",
      "chb10_38_0.npy\n",
      "chb16_17_3.npy\n",
      "chb12_08_1.npy\n",
      "chb20_13_0.npy\n",
      "chb14_27_0.npy\n",
      "chb16_17_2.npy\n",
      "chb01_18_0.npy\n",
      "chb16_16_0.npy\n",
      "chb12_33_0.npy\n",
      "chb14_06_0.npy\n",
      "chb15_54_1.npy\n",
      "chb06_04_1.npy\n",
      "chb15_46_0.npy\n",
      "chb03_02_0.npy\n",
      "chb15_17_0.npy\n",
      "chb12_10_0.npy\n",
      "chb12_36_0.npy\n",
      "chb24_04_1.npy\n",
      "chb15_15_0.npy\n",
      "chb01_26_0.npy\n",
      "chb12_33_1.npy\n",
      "chb10_30_0.npy\n",
      "chb10_31_0.npy\n",
      "chb16_11_0.npy\n",
      "chb12_23_1.npy\n",
      "chb01_16_0.npy\n",
      "chb22_20_0.npy\n",
      "chb20_68_0.npy\n",
      "chb24_01_0.npy\n",
      "chb23_09_0.npy\n",
      "chb13_62_2.npy\n"
     ]
    }
   ],
   "source": [
    "for (kind, file) in zip(labels, basenames):\n",
    "    Xs_original = np.load(os.path.join(data_dir, 'interim', 'X', file))\n",
    "    Ys_original = np.load(os.path.join(data_dir, 'interim', 'Y', file))\n",
    "    Xs_normalized = normalize_feature_vectors(Xs_original)\n",
    "    Xs_normalized = np.concatenate([\n",
    "        Xs_normalized[:, :585],\n",
    "        Xs_normalized[:, 603:639],\n",
    "        Xs_normalized[:, 640:]\n",
    "    ], axis=1)\n",
    "    print(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
